#include <linux/skbuff.h>
#include <linux/if_arp.h>
#include <linux/netdevice.h>
#include <linux/if.h>
#include <linux/if_vlan.h>
#include <net/udp_tunnel.h>
#include <net/sch_generic.h>
#include <linux/netfilter.h>
#include <rdma/ib_addr.h>

#include "rxe.h"
#include "rxe_net.h"

/*内核态套接字*/
static struct rxe_recv_sockets recv_sockets;

int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
{
    int err;
    unsigned char ll_addr[ETH_ALEN];

    ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);
    err = dev_mc_add(rxe->ndev, ll_addr);

    return err;
}

int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid)
{
    int err;
    unsigned char ll_addr[ETH_ALEN];

    ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr);
    err = dev_mc_del(rxe->ndev, ll_addr);

    return err;
}

static struct dst_entry *rxe_find_route4(struct net_device *ndev,
        struct in_addr *saddr, struct in_addr *daddr)
{
    struct rtable *rt;
    struct flowi4 fl = { { 0 } };

    return &rt->dst;
}

#if IS_ENABLED(CONFIG_IPV6)
static struct dst_entry *rxe_find_route6(struct net_device *ndev,
        struct in6_addr *saddr, struct in6_addr *daddr)
{
    struct dst_entry *ndst;
    struct flowi6 fl6 = { { 0 } };


    return ndst;
}
#else
static struct dst_entry *rxe_find_route6(struct net_device *ndev,
        struct in6_addr *saddr, struct in6_addr *daddr)
{
    return NULL;
}

#endif
static struct dst_entry *rxe_find_route(struct net_device *ndev,
        struct rxe_qp *qp, struct rxe_av *av)
{
    struct dst_entry *dst = NULL;

    return dst;
}

static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
{
    struct udphdr *udph;
    struct rxe_dev *rxe;
    struct net_device *ndev = skb->dev;
    /*指向skb的私有数据字段*/
    struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);

    rxe = rxe_get_dev_from_net(ndev);
    if (!rxe && is_vlan_dev(ndev))
        rxe = rxe_get_dev_from_net(vlan_dev_real_dev(ndev));
    if (!rxe)
        goto drop;

    //https://blog.csdn.net/zhuchao576991830/article/details/39895963/
    //判断当前skb是否被分段，有分段的话，把分段的数据合并到当前的主buff来
    if (skb_linearize(skb)) {
        pr_err("skb_linearize failed\n");
        ib_device_put(&rxe->ib_dev);
        goto drop;
    }

    //指向udp头的位置
    udph = udp_hdr(skb);
    pkt->rxe = rxe;
    pkt->port_num = 1;
    //跳过udp头，现在执行infiniband的位置
    pkt->hdr = (u8 *)(udph + 1);
    pkt->mask = RXE_GRH_MASK;
    //upd,len字段减去udp长度等于负载长度,BTH开始后面的长度
    pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);

    rxe_rcv(skb);

    return 0;
drop:
    kfree_skb(skb);
    return 0;
}

static struct socket *rxe_setup_udp_tunnel(struct net *net,
        __be16 port, bool ipv6)
{
    int err;
    struct socket *sock;
    struct udp_port_cfg udp_cfg = {};
    struct udp_tunnel_sock_cfg tnl_cfg = {};

    if (ipv6) {
        udp_cfg.family = AF_INET6;
        udp_cfg.ipv6_v6only = 1;
    } else {
        udp_cfg.family = AF_INET;
    }

    udp_cfg.local_udp_port = port;
    err = udp_sock_create(net, &udp_cfg, &sock);
    if(err  < 0)
        return ERR_PTR(err);

    tnl_cfg.encap_type = 1;
    tnl_cfg.encap_rcv = rxe_udp_encap_recv; /*转发目的端口为4791的udp包到这个函数里*/

    setup_udp_tunnel_sock(net, sock, &tnl_cfg);

    return sock;
}

static void rxe_release_udp_tunnel(struct socket *sk)
{
    if(sk)
        udp_tunnel_sock_release(sk);
}

static void prepare_udp_hdr(struct sk_buff *skb, __be16 src_port,
        __be16 dst_port)
{
    struct udphdr *udph;

    //skbuff.h-->data指针往上移动
    __skb_push(skb, sizeof(*udph));

    //找到upd头的起始地址
    skb_reset_transport_header(skb);
    udph = udp_hdr(skb);

    //udp头的4个字段赋值
    udph->dest = dst_port;
    udph->source = src_port;
    //UDP数据报的长度（包括数据和首部），其最小值为8B（即仅有首部没有数据的情况）
    udph->len = htons(skb->len);//这个值不包括数据部分和icrc
    udph->check = 0;
}

//https://blog.csdn.net/weixin_39770311/article/details/116773970
static void prepare_ipv4_hdr(struct dst_entry *dst, struct sk_buff *skb,
         __be32 saddr, __be32 daddr, __u8 proto,
        __u8 tos, __u8 ttl, __be16 df, bool xnet)
{
    struct iphdr *iph;

    /*清空skb里面一些其它字段的值*/
    skb_scrub_packet(skb, xnet);
    skb_clear_hash(skb);
    skb_dst_set(skb, dst_clone(dst));
    memset(IPCB(skb), 0, sizeof(*IPCB(skb)));

    /*指向ip头起始地址处--https://blog.csdn.net/tryheart/article/details/107547074*/
    skb_push(skb, sizeof(struct iphdr));//预留ipv4头
    skb_reset_network_header(skb);
    iph = ip_hdr(skb);

    /*4bit的版本号*/
    iph->version = IPVERSION;
    //头部长度，范围20-60B(由于是4个bit表示，所以这里的值还要乘以4)
    iph->ihl = sizeof(struct iphdr) >> 2;//5*4Byte
    //服务类型,traffic_class字段表示
    iph->tos = tos;
    //16位总长度,首部加数据
    iph->tot_len = htons(skb->len);//ip+udp+rdma,不包括数据和icrc
    /*16位标识，表示IP数据报的标识符，占16位，
        每个IP数据报有一个唯一的标识。当数据报超过下面链路层的MTU值必须分段时，
        这个表示字段的值被复制到所有的数据报的标识字段中；
        相同的标识字段的值使分段后的各数据报分段最后能正确的重装成为原来的数据报*/
    __ip_select_ident(dev_net(dst->dev), iph,
        skb_shinfo(skb)->gso_segs ?: 1);
    /*目前只有前两位有意义,最低位为1表示后面还有分片的数据报,
    为0表示这已经是最后一个数据报，中间一位为1表示不能分片，为0才允许分片*/
    iph->frag_off = df;//0x4000
    //hop_limit字段表示
    iph->ttl = ttl;
    //指该数据包是使用什么协议,当前是UDP协议
    iph->protocol = proto;
    //源地址，目的地址
    iph->saddr = saddr;
    iph->daddr = daddr;
    return;
}

static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb,
        struct in6_addr *saddr, struct in6_addr *daddr,
        __u8 proto, __u8 prio, __u8 ttl)
{
    //...
    return;
}

static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
    struct rxe_qp *qp = pkt->qp;
    struct dst_entry *dst;
    bool xnet = false;
    __be16 df = htons(IP_DF); /*不分片*/
    struct rxe_av *av = rxe_get_av(pkt);

    //ipv4地址
    struct in_addr *saddr = &av->sgid_addr._sockaddr_in.sin_addr;
    struct in_addr *daddr = &av->dgid_addr._sockaddr_in.sin_addr;

    //主要作用是返回一个指针，struct dst_entry
    dst = rxe_find_route(skb->dev, qp, av);
    if (!dst) {
        pr_err("Host not reachable\n");
        return -EHOSTUNREACH;
    }

    /*UDP头填充(8B-字节)*/
    prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
        cpu_to_be16(ROCE_V2_UDP_DPORT));

    /*IPV4头部字段填充*/
    prepare_ipv4_hdr(dst, skb, saddr->s_addr, daddr->s_addr, IPPROTO_UDP,
        av->grh.traffic_class, av->grh.hop_limit, df, xnet);

    //释放
    dst_release(dst);
    return 0;
}

static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
    struct rxe_qp *qp = pkt->qp;
    struct dst_entry *dst;
    struct rxe_av *av = rxe_get_av(pkt);
    //ipv6地址
    struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr;
    struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr;

    //找到struct dst_entry结构体指针
    dst = rxe_find_route(skb->dev, qp, av);
    if (!dst) {
        pr_err("Host not reachable\n");
        return -EHOSTUNREACH;
    }
    /*udp头填充*/
    prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
        cpu_to_be16(ROCE_V2_UDP_DPORT));
    /*IPV6头填充*/
    prepare_ipv6_hdr(dst, skb, saddr, daddr, IPPROTO_UDP,
            av->grh.traffic_class, av->grh.hop_limit);

    dst_release(dst);
    return 0;
}

int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb)
{
    int err = 0;

    if(skb->protocol == htons(ETH_P_IP))
        err = prepare4(pkt, skb);
    else if(skb->protocol == htons(ETH_P_IPV6))
        err = prepare6(pkt, skb);

    /*判断是否是loop包*/
    if (ether_addr_equal(skb->dev->dev_addr, rxe_get_av(pkt)->dmac))
        pkt->mask |= RXE_LOOPBACK_MASK;

    return err;
}

static void rxe_skb_tx_dtor(struct sk_buff *skb)
{
    struct sock *sk = skb->sk;
    struct rxe_qp *qp = sk->sk_user_data;

    //...
}

static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
{
    int err;

    skb->destructor = rxe_skb_tx_dtor;
    /*sock*/
    skb->sk = pkt->qp->sk->sk;

    rxe_add_ref(pkt->qp);
    atomic_inc(&pkt->qp->skb_out);

    if (skb->protocol == htons(ETH_P_IP)) {
        err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
    } else if (skb->protocol == htons(ETH_P_IPV6)) {
        err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
    } else {
        pr_err("Unknown layer 3 protocol: %d\n", skb->protocol);
        atomic_dec(&pkt->qp->skb_out);
        rxe_drop_ref(pkt->qp);
        kfree_skb(skb);
        return -EINVAL;
    }

    /*判断err返回值*/
    if (unlikely(net_xmit_eval(err))) {
        pr_debug("error sending packet: %d\n", err);
        return -EAGAIN;
    }

    return 0;
}

static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
{
    //把pkt信息存储到cb里
    memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt));

    /*移动data指针，相当于丢弃ip头*/
    if (skb->protocol == htons(ETH_P_IP))
        skb_pull(skb, sizeof(struct iphdr));
    else
        skb_pull(skb, sizeof(struct ipv6hdr));

    /*检查有没有收包的设备*/
    if (WARN_ON(!ib_device_try_get(&pkt->rxe->ib_dev))) {
        kfree_skb(skb);
        return -EIO;
    }

    //这个时候udp头还在
    rxe_rcv(skb);
    return 0;
}

int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
        struct sk_buff *skb)
{
    int err;
    int is_request = pkt->mask & RXE_REQ_MASK;
    struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

    /*这个判断还不是很理解?*/
    if ((is_request && (qp->req.state != QP_STATE_READY)) ||
         (!is_request && (qp->resp.state != QP_STATE_READY))) {
        pr_info("Packet dropped. QP is not in ready state\n");
        goto drop;
    }

    /*在数据包的最后增加icrc4*/
    rxe_icrc_generate(skb, pkt);

    if (pkt->mask & RXE_LOOPBACK_MASK)
        err = rxe_loopback(skb, pkt);
    else
        err = rxe_send(skb, pkt);
    if (err) {
        rxe->xmit_errors++;
        //发包错误计数
        rxe_counter_inc(rxe, RXE_CNT_SEND_ERR);
        return err;
    }
    //发包计数
    rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
    goto done;

drop:
    kfree_skb(skb);
    err = 0;
done:
    return err;
}

struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
        int paylen, struct rxe_pkt_info *pkt)
{
    unsigned int hdr_len; /*L2+L3+L4*/
    struct sk_buff *skb = NULL;
    struct net_device *ndev;
    const struct ib_gid_attr *attr;
    const int port_num = 1;

    attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index);
    if (IS_ERR(attr))
        return NULL;

    /*根据ip类型求hdr_len长度*/
    if (av->network_type == RXE_NETWORK_TYPE_IPV4)
        hdr_len = ETH_HLEN + sizeof(struct udphdr) + sizeof(struct iphdr);
    else
        hdr_len = ETH_HLEN + sizeof(struct udphdr) + sizeof(struct ipv6hdr);

    rcu_read_lock();
    ndev = rdma_read_gid_attr_ndev_rcu(attr);
    if (IS_ERR(ndev)) {
        rcu_read_unlock();
        goto out;
    }

    /*skb->head = skb->data =
        skb->tail, skb->len=0 skb->end = skb->tail + size
        LL_RESERVED_SPACE(ndev) = 16
    */
    skb = alloc_skb(paylen + hdr_len + LL_RESERVED_SPACE(ndev), GFP_ATOMIC);
    if(unlikely(!skb)) {
        rcu_read_unlock();
        goto out;
    }

    /*skb->data = skb->data + size
        skb->tail = skb->tail + size */
    skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev));

    skb->dev = ndev;
    rcu_read_unlock();

    pkt->rxe = rxe;
    pkt->port_num = port_num;
    pkt->hdr = skb_put(skb, paylen); //现在指向L4起始地址，skb->tail = skb->tail + paylen;
    pkt->mask |= RXE_GRH_MASK;
out:
    rdma_put_gid_attr(attr);
    return skb;
}

const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num)
{
    return rxe->ndev->name;
}

int rxe_net_add(const char * ibdev_name, struct net_device * ndev)
{
    int err;
    struct rxe_dev *rxe = NULL;

    rxe = ib_alloc_device(rxe_dev, ib_dev);
    if(!rxe)
        return -ENOMEM;

    rxe->ndev = ndev;

    err = rxe_add(rxe, ndev->mtu, ibdev_name);
    if(err) {
        ib_dealloc_device(&rxe->ib_dev);
        return err;
    }
    return 0;
}

static void rxe_port_event(struct rxe_dev *rxe,
        enum ib_event_type event)
{
    struct ib_event ev;

    ev.device = &rxe->ib_dev;
    ev.element.port_num = 1;
    ev.event = event;

    ib_dispatch_event(&ev);
}

void rxe_port_up(struct rxe_dev *rxe)
{
    struct rxe_port *port;

    port = &rxe->port;
    port->attr.state = IB_PORT_ACTIVE;

    rxe_port_event(rxe, IB_EVENT_PORT_ACTIVE);
    dev_info(&rxe->ib_dev.dev, "set active\n");
}

void rxe_port_down(struct rxe_dev *rxe)
{
    struct rxe_port *port;

    port = &rxe->port;
    port->attr.state = IB_PORT_DOWN;

    rxe_port_event(rxe, IB_EVENT_PORT_ERR);
    rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED);
    dev_info(&rxe->ib_dev.dev, "set down\n");
}

/*设置端口状态*/
void rxe_set_port_state(struct rxe_dev *rxe)
{
    if (netif_running(rxe->ndev) && netif_carrier_ok(rxe->ndev))
        rxe_port_up(rxe);
    else
        rxe_port_down(rxe);
}

static int rxe_notify(struct notifier_block *not_blk,
        unsigned long event, void *arg)
{
    struct net_device *ndev = netdev_notifier_info_to_dev(arg);
    struct rxe_dev *rxe = rxe_get_dev_from_net(ndev);

    if(!rxe)
        return NOTIFY_OK;

    switch(event) {
    case NETDEV_UNREGISTER:
        ib_unregister_device_queued(&rxe->ib_dev);
        break;
    case NETDEV_UP:
        rxe_port_up(rxe);
        break;
    case NETDEV_DOWN:
        rxe_port_down(rxe);
        break;
    case NETDEV_CHANGEMTU:
        pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
        rxe_set_mtu(rxe, ndev->mtu);
        break;
    case NETDEV_CHANGE:
        rxe_set_port_state(rxe);
        break;
    case NETDEV_REBOOT:
    case NETDEV_GOING_DOWN:
    case NETDEV_CHANGEADDR:
    case NETDEV_CHANGENAME:
    case NETDEV_FEAT_CHANGE:
    default:
        pr_info("ignoring netdev event = %ld for %s\n", event, ndev->name);
        break;
    }

    ib_device_put(&rxe->ib_dev);
    return return NOTIFY_OK;
}

static struct notifier_block rxe_net_notifier = {
    .notifier_call = rxe_notify,
};

static int rxe_net_ipv4_init(void)
{
    /*创建一个ipv4套接字*/
    recv_sockets.sk4 = rxe_setup_udp_tunnel(&init_net,
        htons(ROCE_V2_UDP_DPORT), false);
    if (IS_ERR(recv_sockets.sk4)) {
        recv_sockets.sk4 = NULL;
        pr_err("Failed to create IPv4 UDP tunnel\n");
        return -1;
    }
    return 0;
}

static int rxe_net_ipv6_init(void)
{
#if IS_ENABLED(CONFIG_IPV6)
    /*最后一个为true表示创建ipv6套接字*/
    recv_sockets.sk6 = rxe_setup_udp_tunnel(&init_net,
            htons(ROCE_V2_UDP_DPORT), true);
    if (PTR_ERR(recv_sockets.sk6) == -EAFNOSUPPORT) {
        recv_sockets.sk6 = NULL;
        pr_warn("IPv6 is not supported, can not create a UDPv6 socket\n");
        return 0;
    }
    if (IS_ERR(recv_sockets.sk6)) {
        recv_sockets.sk6 = NULL;
        pr_err("Failed to create IPv6 UDP tunnel\n");
        return -1;
    }
#endif
    return 0;
}

void rxe_net_exit(void)
{
    rxe_release_udp_tunnel(recv_sockets.sk6);
    rxe_release_udp_tunnel(recv_sockets.sk4);
    unregister_netdevice_notifier(&rxe_net_notifier);
}

int rxe_net_init(void)
{
    int err;
    recv_sockets.sk6 = NULL;

    err = rxe_net_ipv4_init();
    if(err)
        return err;

    err = rxe_net_ipv6_init();
    if(err)
        goto err;

    err = register_netdevice_notifier(&rxe_net_notifier);
    if(err) {
        pr_err("Failed to register netdev notifier\n");
        goto err;
    }
    return 0;
err:
    rxe_net_exit();
    return err;
}
